
from scrapy.spiders import Spider
import scrapy
from scrapy_splash import SplashRequest
from scrapy.selector import Selector
from reedlive_scrapy.items import WeChatItem
from shortid import ShortId
from reedlive_scrapy.utils.dbmanager import DbManager
import time
from dateutil import parser
import re



class WeChat(Spider):
    name = 'wechat_test'

    def start_requests(self):
        self.dbt = DbManager()

        yield SplashRequest('http://chuansong.me/account/idxgh2013',
        # yield SplashRequest('http://chuansong.me/n/1499449941121',
                             args={'wait': 0.5},
                             callback=self.getWeChatItems,
                             errback=self.parse_error,
                             meta={"cookiejar": 1}
                             )

    def getWeChatItems(self,response):
        # print response.body
        channel = self.dbt.getChannelFromUrl(response.url)
        itemSet=self.parse_wechat(response,channel['_id'])
        # itemSet=self.parse_wechat(response,'ddd')

        itemCount=0
        syncSet = set()
        for item in itemSet:
            if not self.dbt.isIdExist(item['wid']):
                itemCount=itemCount+1
                syncSet.add(item)
                yield item

        if itemCount != 0:
            channel['updateTime'] = long(time.time())
            self.dbt.updateChannelTime(channel)
            base_url='http://chuansong.me'
            for item in syncSet:
                time.sleep(5)
                itemurl=base_url+'/n/'+item['wid']
                print itemurl
                yield scrapy.Request(itemurl,
                                     meta={
                                         'dont_redirect': True,
                                         'handle_httpstatus_list': [301, 302]
                                     },
                                     headers={'wid':item['_id']},
                                     dont_filter=True,
                                     callback=self.updateWeChatContent,
                                     errback=self.parse_error
                                     )


    def parse_wechat(self,response,channelid):
        selector = Selector(text=response.body)
        nodes = selector.xpath("//div[@class='feed_item_question']")
        itemsSet = set()
        for node in nodes:
            item=WeChatItem()
            hre=node.xpath("h2/span/a")
            mTitle=hre.xpath("text()").extract_first().encode('utf8')
            wid=hre.xpath("@href").extract_first()
            if len(wid)>4:
                wid=wid[3:]
            dat=node.xpath("h2/span/span[@class='timestamp']/text()").extract_first()


            mDate_time = parser.parse(dat)
            mpub = time.mktime(mDate_time.timetuple())
            mTime = time.time()

            item['_id'] = ShortId().generate()
            item['wid'] = wid
            item['title'] = mTitle
            item['published'] = long(mpub)
            item['updated'] = long(mTime)
            item['channelId'] = channelid
            itemsSet.add(item)
        return itemsSet


    def updateWeChatContent(self,response):
        print 'success    '+response.url
        selector = Selector(text=response.body)
        content=selector.xpath("//div[@class='rich_media_content']/text()").extract_first()
        dr = re.compile(r'<[^>]+>', re.S)
        mSubTitle = dr.sub('', content).strip()
        if len(mSubTitle) > 60:
            mSubTitle = mSubTitle[0:59]

        textCount = self.GetWordNums(dr.sub('', content).strip())

        self.dbt.updateTuikuItem(response.request.headers['wid'],content,mSubTitle,textCount)


    def parse_error(self,error):
        print error


    def GetWordNums(self, text):
        num = 0
        for i in text:
            if i not in ' \n!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~"':
                num = num + 1
        return num
